package com.cloud.spider;

import com.cloud.controller.GitHubController;
import com.cloud.entity.RepositoryItem;
import com.cloud.entity.RespVo;
import com.cloud.entity.RepositoryItem;
import com.google.gson.Gson;

import java.io.*;
import java.util.*;

public class PreProcess {

    List<List<String>> words = new ArrayList<>();

    public void process(String path) throws Exception {
        File file = new File(path);
        process(file);
    }

    public void process(File file) throws Exception {
        BufferedReader br = new BufferedReader(new FileReader(file));
        StringBuilder sb = new StringBuilder();
        String line = br.readLine();
        while (line != null) {
            sb.append(line);
            line = br.readLine();
        }
        String body = sb.toString();
        Gson gson = new Gson();
        RespVo respVo = gson.fromJson(body, RespVo.class);
        for (RepositoryItem item : respVo.getItems()) {
            words.add(item.getTopics());
        }
    }

    public List<List<String>> getWords() {
        return words;
    }

    private void preprocessEachLang2WcFile(String lang) throws Exception {
        words.clear();
        File dir = new File("data/" + lang);
        if (!dir.exists()) return;
        for (File file : dir.listFiles()) {
            process(file);
        }
        List<List<String>> words = getWords();
        File goTxt = new File("data/wcinput/" + lang + "/data.txt");
        File parentLangDir = new File("data/wcinput/" + lang);
        if (!parentLangDir.exists()) parentLangDir.mkdirs();
        if (!goTxt.exists()) goTxt.createNewFile();
        BufferedWriter bw = new BufferedWriter(new FileWriter(goTxt));
        for (List<String> word : words) {
            if (word.size() == 0) continue;
            for (String s : word) {
                bw.write(s + " ");
            }
            bw.newLine();
        }
        bw.flush();
    }

    private void preprocessEachLang(String lang) throws Exception {
        words.clear();
        File dir = new File("data/" + lang);
        HashMap<String, Integer> map = new HashMap<>();
        for (File data : dir.listFiles()) {
            process(data);
        }
        System.out.println(lang + ":" + words.size());
        for (List<String> word : getWords()) {
            for (String w : word) {
                map.put(w, map.getOrDefault(w, 0) + 1);
            }
        }
        List<Map.Entry<String, Integer>> list = new ArrayList<>();
        for (Map.Entry<String, Integer> entry : map.entrySet()) {
            list.add(entry);
        }
        Collections.sort(list, new Comparator<Map.Entry<String, Integer>>() {
            @Override
            public int compare(Map.Entry<String, Integer> o1, Map.Entry<String, Integer> o2) {
                return o2.getValue() - o1.getValue();
            }
        });
        File goTxt = new File("data/wordcount/" + lang + ".txt");
        BufferedWriter bw = new BufferedWriter(new FileWriter(goTxt));
        for (Map.Entry<String, Integer> entry : list) {
            bw.write(String.format("%s %d", entry.getKey(), entry.getValue()));
            bw.newLine();
        }
        bw.flush();
        bw.close();
    }


    static String[] langs = GitHubController.langs;

    public static void main(String[] args) throws Exception {
        PreProcess process = new PreProcess();
        for (String lang : langs) {
            process.preprocessEachLang2WcFile(lang.toLowerCase());
        }
        for (String lang : langs) {
            process.preprocessEachLang(lang.toLowerCase());
        }
    }
}
